
/*------------------------------------------------------------------------
 *  Copyright (C) 2010  Luis M. de la Cruz
 *
 *  This file is part of TUNA
 *
 *  TUNA is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  TUNA is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ------------------------------------------------------------------------*/

#ifndef _CARTCOMM_HPP_
#define _CARTCOMM_HPP_

#include <iostream>
#include "Tuna.hpp"
#include <mpi.h>

namespace Tuna {

/*!
 ***************************************************************************
 * 2D Catersian communicator in the MPI environment. 
 * This class is used to construct a Cartesian topology to be used in simple
 * rectangular-like domains. It is supposed all subdomains that belongs to
 * this topology are rectangles as well. The \c comm variable is an 
 * MPI::Cartcomm communicator, and to be able to access all its functionality 
 * from outside the class in an easy way, this variable is left public. The
 * orientation of the axes (x,y,z) and the numeration of the processors 
 * inside the topology is as follows:
 * \verbatim    

   ^ y-axis (physical domain)
   |
   +---------+---------+---------+---------+
   |         |         |         |         |   Number of Processors = 12
 ^ |    2    |    5    |    8    |   11    |   
 | |  (0,2)  |  (1,2)  |  (2,2)  |  (3,2)  |   rank = 0,1,2,3,4,5,6,7,8,9,10,11
 | |         |         |         |         |          
 | +---------+---------+---------+---------+   
   |         |         |         |         |   I = 0,1,2,3 
   |    1    |    4    |    7    |   10    |   J = 0,1,2
 J |  (0,1)  |  (1,1)  |  (2,1)  |  (3,1)  |   
   |         |         |         |         |   
 | +---------+---------+---------+---------+   
 | |         |         |         |         |   
 | |    0    |    3    |    6    |    9    |   
 v |  (0,0)  |  (1,0)  |  (2,0)  |  (3,0)  |   
   |         |         |         |         |   
   +---------+---------+---------+---------+---> x-axis (physical domain)
                 <---- I ---->
   \endverbatim   
 * The above orientation and numeration is obtained defining 
 * \f x \simm \f [LEFT, RIGHT] ,  \f y \simm \f [UP, DOWN] and  
 * \f z \simm \f [FRONT, BACK] :
 * \verbatim    
                       UP (I, J+1) 
			    ^
                            |

      LEFT (I-1, J) <---  (I,J)  ---> RIGHT (I-1, J)

                            |
		            v	
                       DOWN (I, J-1)     
   \endverbatim
 *  
 * \todo Complete with a figure made in xfig.
 ***************************************************************************
 *  \author  Luis M. de la Cruz [ Tue Mar 18 11:10:23 GMT 2008 ]
 ***************************************************************************
 */  
template<int Dim>
class CartComm
{
public:
    CartComm() { };    
    CartComm(int argc, char **argv, int r);
    
    inline int getNumProc_I() { return num_proc[0]; }
    inline int getNumProc_J() { return num_proc[1]; }
    inline int getNumProc_K() { return num_proc[2]; }
    inline int get_I() { return coords[0]; }    
    inline int get_J() { return coords[1]; }
    inline int get_K() { return coords[2]; }
    inline int get_rank() { return rank; }
    inline int getNeighborProc(cartDir_t n) { return neighbors[n]; }
    inline const int* getNeighbors() { return neighbors; }


    void print () {

      std::cout << "\n +----- TUNA info -----+"
		<< "\n | CartComm < " << Dim << "  > "
		<< "\n +-----+";

      if (Dim == 1)
	std::cout << "\n | Proc " << rank 
		  << " : IJK (" << coords[0] << ") "
		  << " | RIGHT = " << neighbors[RIGHT]
		  << " | LEFT = " << neighbors[LEFT];

      if (Dim == 2)
	std::cout << "\n | Proc " << rank 
		  << " : IJK (" << coords[0] << "," << coords[1] << ") "
		  << " | RIGHT = " << neighbors[RIGHT]
		  << " | LEFT = " << neighbors[LEFT]	  
		  << " | UP = " << neighbors[UP]
		  << " | DOWN = " << neighbors[DOWN];

      if (Dim == 3)
	std::cout << "\n | Proc " << rank 
		  << " : IJK (" << coords[0] << "," << coords[1] << "," << coords[2] << ") "
 		  << " | RIGHT = " << neighbors[RIGHT]
		  << " | LEFT = " << neighbors[LEFT]	  
		  << " | UP = " << neighbors[UP]
		  << " | DOWN = " << neighbors[DOWN]
		  << " | FRONT = " << neighbors[FRONT]
		  << " | BACK = " << neighbors[BACK];

      std::cout << "\n +---------------------+\n "; 
    }
  
  //  MPI::Cartcomm comm; ///< Communicator for a Cartesian topology.
  MPI_Comm comm; ///< Communicator for a Cartesian topology.
  
private:
  int num_proc[3];
  int coords[3];
  int neighbors[6];
  int periods[3];   // bool periods[3]; //in C++
  int rank;
};

/*!
 *  The Cartesian topology is constructed here. 
 */
template<int Dim>
CartComm<Dim>::CartComm(int argc, char **argv, int r)
{

  rank = r;    
  
  if (argc == 4) { // We require 3 arguments plus the name of the executable
    num_proc[0] = std::atoi(argv[1]); 
    num_proc[1] = std::atoi(argv[2]); 
    num_proc[2] = std::atoi(argv[3]); 
  } else if (rank == 0) {
    std::cerr << "\n\n DD::CartComm : usage: mpirun -np N " 
	      << argv[0] << " ns_x ns_y ns_z" 
	      << "\n ns_x : number of subdomains in x-axis"
	      << "\n ns_y : number of subdomains in y-axis (use 1 if Dim == 1)"
	      << "\n ns_z : number of subdomains in z-axis (use 1 if Dim == 2)"
	      << "\n N : total number of subdomains N = ns_x * ns_y * ns_z \n\n";
    std::exit(0);
  }
  
  // Array to define the peridicity: NON PERIODIC
  periods[0] = 0; // false
  periods[1] = 0; // false
  periods[2] = 0; // false
  
  //
  // Create de Cart Communicator in 1D
  //
  if (Dim == 1) {
    //    comm = MPI::COMM_WORLD.Create_cart(1, num_proc, periods, true);
    MPI_Cart_create(MPI_COMM_WORLD, 1, num_proc, periods, true, &comm);

    // Find your position in the cart and store it in coords array
    //    comm.Get_coords(rank, 1, coords);     
    MPI_Cart_coords(comm, rank, 1, coords);
    
    // Find your neighbors up, down, left and right, and store it in neighbors array
    // NOTE: if there is not neigbor in some direction, the result is negative.
    //    comm.Shift(0, 1, neighbors[LEFT], neighbors[RIGHT]);
    MPI_Cart_shift(comm, 0, 1, &neighbors[LEFT], &neighbors[RIGHT]);
  }

  //
  // Create de Cart Communicator in 2D
  //
  if (Dim == 2) {
    //    comm = MPI::COMM_WORLD.Create_cart(2, num_proc, periods, true);
    MPI_Cart_create(MPI_COMM_WORLD, 2, num_proc, periods, true, &comm);
    
    // Find your position in the cart and store it in coords array
    //    comm.Get_coords(rank, 2, coords);
    MPI_Cart_coords(comm, rank, 2, coords);
    
    // Find your neighbors up, down, left and right, and store it in neighbors array
    // NOTE: if there is not neigbor in some direction, the result is negative.
    //    comm.Shift(0, 1, neighbors[LEFT], neighbors[RIGHT]);
    //    comm.Shift(1, 1, neighbors[DOWN], neighbors[UP]);
    MPI_Cart_shift(comm, 0, 1, &neighbors[LEFT], &neighbors[RIGHT]);
    MPI_Cart_shift(comm, 1, 1, &neighbors[DOWN], &neighbors[UP]);
  }


  //
  // Create de Cart Communicator in 3D
  //    
  if (Dim == 3) {
    //    comm = MPI::COMM_WORLD.Create_cart(3, num_proc, periods, true);
    MPI_Cart_create(MPI_COMM_WORLD, 2, num_proc, periods, true, &comm);

    // Find your position in the cart and store it in coords array
    //    comm.Get_coords(rank, 3, coords);
    MPI_Cart_coords(comm, rank, 3, coords);
    
    // Find your neighbors up, down, left and right, and store it in neighbors array
    // NOTE: if there is not neigbor in some direction, the result is negative.
    //    comm.Shift(0, 1, neighbors[LEFT], neighbors[RIGHT]);
    //    comm.Shift(1, 1, neighbors[DOWN], neighbors[UP]);
    //    comm.Shift(2, 1, neighbors[BACK], neighbors[FRONT]); 
    MPI_Cart_shift(comm, 0, 1, &neighbors[LEFT], &neighbors[RIGHT]);
    MPI_Cart_shift(comm, 1, 1, &neighbors[DOWN], &neighbors[UP]);
    MPI_Cart_shift(comm, 2, 1, &neighbors[BACK], &neighbors[FRONT]);   
  }
   
}

} // Tuna namespace

#endif // _CARTCOMM_HPP_
